package com.uusee.crawler.job;

import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import com.uusee.crawler.dbwriter.UgcDbWriter;
import com.uusee.crawler.extractor.LinkExtractor;
import com.uusee.crawler.fetcher.FetchHTTP;
import com.uusee.crawler.model.CrawlURI;
import com.uusee.crawler.pageprocessor.video.PlayPageProcessor;
import com.uusee.crawler.xmlwriter.UgcXmlWriter;
import com.uusee.shipshape.sp.model.Ugc;
import com.uusee.util.cache.CacheUtil;

public class TudouNewUgcCrawlJob {
	private String[] categoryCodes = {"-1","22","1","99","9","5","14","10","15","3","21"};
	private String[] typeCodes = {"l","r","v"};
	private String[] sortCodes = {"1","2","3","4"};
	
	private FetchHTTP fetch = new FetchHTTP();
	private PlayPageProcessor pageProcessor = new PlayPageProcessor();
	private UgcXmlWriter xmlwriter = new UgcXmlWriter();
	private UgcDbWriter dbWriter = new UgcDbWriter();
	
	private LinkExtractor linkExtractor = new LinkExtractor("<a href=\"(http://www.tudou.com/programs/view/[^<]*?)\"[^<]*?>");
	
	
	Set<CrawlURI> categoryLinks = new HashSet<CrawlURI>();
	
	private CacheUtil<CrawlURI> tudouCrawlURICache;
	
	public static void main(String[] args) {
		try {
			ApplicationContext acx = new ClassPathXmlApplicationContext("applicationContext-ugc.xml");
			TudouNewUgcCrawlJob tudouNewUgcCrawlJob = (TudouNewUgcCrawlJob)acx.getBean("tudouNewUgcCrawlJob");
			tudouNewUgcCrawlJob.doExecute();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	public void doExecute() {
		try {
			for (String type:typeCodes) {
				for (String cate:categoryCodes) {
					if (type.equals("v")) {
						for (String sort:sortCodes) {
							for (int p=1;p<=30;p++) {
								String crawlUrl = "http://www.tudou.com/top/"+type+"c"+cate+"p"+p+"p4o"+sort+".html";
								CrawlURI cu = new CrawlURI();
								cu.setCrawlUrl(crawlUrl);
								cu.setCharset("gbk");
								cu.setSourceSite("www.tudou.com");
								
								categoryLinks.add(cu);
							}
						}
					} else {
						for (int p=1;p<=30;p++) {
							String crawlUrl = "http://www.tudou.com/top/"+type+"c"+cate+"p"+p+".html";
							CrawlURI cu = new CrawlURI();
							cu.setCrawlUrl(crawlUrl);
							cu.setCharset("gbk");
							cu.setSourceSite("www.tudou.com");
							
							categoryLinks.add(cu);
						}
					}
				}
			}
			
			
			System.out.println(categoryLinks.size());
			for (CrawlURI categoryCrawlURI:categoryLinks) {
				System.out.println("抓取："+categoryCrawlURI.getCrawlUrl());
				fetch.process(categoryCrawlURI);
				linkExtractor.process(categoryCrawlURI);
				Set<CrawlURI> ugcLinks = (Set<CrawlURI>)categoryCrawlURI.getModel();
				for (CrawlURI ugcLinkCrawlURI:ugcLinks) {
					Ugc ugc = new Ugc();
					ugc.setVideoSite("www.tudou.com");
					
					ugcLinkCrawlURI.setModel(ugc);
					ugcLinkCrawlURI.setCharset("gbk");
					
					if (!tudouCrawlURICache.containsKey(ugcLinkCrawlURI)) {
						tudouCrawlURICache.put(ugcLinkCrawlURI,ugcLinkCrawlURI);
						System.out.println("抓取："+ugcLinkCrawlURI.getCrawlUrl());
						crawl(ugcLinkCrawlURI);
					} 
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	
	
	public void crawl(CrawlURI crawURI) {
		try {
				fetch.process(crawURI);
				pageProcessor.process(crawURI);
				//xmlwriter.process(crawURI);
				dbWriter.process(crawURI);
		} catch(Exception e) {
			e.printStackTrace();
		}
	}

	public void setCategoryCodes(String[] categoryCodes) {
		this.categoryCodes = categoryCodes;
	}

	public void setTypeCodes(String[] typeCodes) {
		this.typeCodes = typeCodes;
	}

	public void setSortCodes(String[] sortCodes) {
		this.sortCodes = sortCodes;
	}

	public void setFetch(FetchHTTP fetch) {
		this.fetch = fetch;
	}

	public void setPageProcessor(PlayPageProcessor pageProcessor) {
		this.pageProcessor = pageProcessor;
	}

	public void setXmlwriter(UgcXmlWriter xmlwriter) {
		this.xmlwriter = xmlwriter;
	}

	public void setDbWriter(UgcDbWriter dbWriter) {
		this.dbWriter = dbWriter;
	}

	public void setLinkExtractor(LinkExtractor linkExtractor) {
		this.linkExtractor = linkExtractor;
	}

	public void setCategoryLinks(Set<CrawlURI> categoryLinks) {
		this.categoryLinks = categoryLinks;
	}

	public void setTudouCrawlURICache(CacheUtil<CrawlURI> tudouCrawlURICache) {
		this.tudouCrawlURICache = tudouCrawlURICache;
	}

	
	
}
